{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bert import tokenization_sentencepiece as tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_file = 'ja-checkpoint/wiki-ja.model'\n",
    "vocab_file = 'ja-checkpoint/wiki-ja.vocab'\n",
    "do_lower_case = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded a trained SentencePiece model.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tokenization.FullTokenizer(model_file, vocab_file, do_lower_case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = \"誕生日特典ポイントは誕生月内に複数の店舗を利用した場合、それぞれ付与されるのか\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = tokenizer.tokenize(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['▁', '誕生日', '特典', 'ポイント', 'は', '誕生', '月', '内に', '複数の', '店舗', 'を利用した', '場合', '、', 'それぞれ', '付与', 'される', 'のか']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(tokens)\n",
    "len(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [0 for _ in range(len(sentence))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords = ['誕生日特典ポイント', '複数の店舗']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = [0 for _ in range(len(tokens))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for keyword in keywords:\n",
    "    start_idx = sentence.find(keyword)\n",
    "    end_idx = start_idx + len(keyword)\n",
    "    # sentence[start_idx:end_idx]\n",
    "    for i in range(start_idx, end_idx):\n",
    "        labels[i] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n"
     ]
    }
   ],
   "source": [
    "index = 0\n",
    "for tid, token in enumerate(tokens):\n",
    "    for char in str(token):\n",
    "        if char == '▁':\n",
    "            continue\n",
    "        if labels[index] == 1:\n",
    "            tags[tid] = 1\n",
    "        index += 1\n",
    "print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "▁ 0\n",
      "誕生日 1\n",
      "特典 1\n",
      "ポイント 1\n",
      "は 0\n",
      "誕生 0\n",
      "月 0\n",
      "内に 0\n",
      "複数の 1\n",
      "店舗 1\n",
      "を利用した 0\n",
      "場合 0\n",
      "、 0\n",
      "それぞれ 0\n",
      "付与 0\n",
      "される 0\n",
      "のか 0\n"
     ]
    }
   ],
   "source": [
    "for token, tag in zip(tokens, tags):\n",
    "    print(token, tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del_idx = tokens.index('▁')\n",
    "tokens.pop(del_idx)\n",
    "tags.pop(del_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "誕生日 1\n",
      "特典 1\n",
      "ポイント 1\n",
      "は 0\n",
      "誕生 0\n",
      "月 0\n",
      "内に 0\n",
      "複数の 1\n",
      "店舗 1\n",
      "を利用した 0\n",
      "場合 0\n",
      "、 0\n",
      "それぞれ 0\n",
      "付与 0\n",
      "される 0\n",
      "のか 0\n"
     ]
    }
   ],
   "source": [
    "for token, tag in zip(tokens, tags):\n",
    "    print(token, tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 誕\n",
      "1 生\n",
      "2 日\n",
      "3 特\n",
      "4 典\n",
      "5 ポ\n",
      "6 イ\n",
      "7 ン\n",
      "8 ト\n",
      "9 は\n",
      "10 誕\n",
      "11 生\n",
      "12 月\n",
      "13 内\n",
      "14 に\n",
      "15 複\n",
      "16 数\n",
      "17 の\n",
      "18 店\n",
      "19 舗\n",
      "20 を\n",
      "21 利\n",
      "22 用\n",
      "23 し\n",
      "24 た\n",
      "25 場\n",
      "26 合\n",
      "27 、\n",
      "28 そ\n",
      "29 れ\n",
      "30 ぞ\n",
      "31 れ\n",
      "32 付\n",
      "33 与\n",
      "34 さ\n",
      "35 れ\n",
      "36 る\n",
      "37 の\n",
      "38 か\n"
     ]
    }
   ],
   "source": [
    "for i, char in enumerate(sentence):\n",
    "    print(i, char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['▁\\t0',\n",
       " '誕生日\\t1',\n",
       " '特典\\t1',\n",
       " 'ポイント\\t1',\n",
       " 'は\\t0',\n",
       " '誕生\\t0',\n",
       " '月\\t0',\n",
       " '内に\\t0',\n",
       " '複数の\\t1',\n",
       " '店舗\\t1',\n",
       " 'を利用した\\t0',\n",
       " '場合\\t0',\n",
       " '、\\t0',\n",
       " 'それぞれ\\t0',\n",
       " '付与\\t0',\n",
       " 'される\\t0',\n",
       " 'のか\\t0']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "['\\t'.join([token, str(tag)]) for token, tag in zip(tokens, tags)]\n",
    "# [token for token in tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'の0'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([token[0], str(tags[0])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
